'''
Description: 
Autor: didiplus
Date: 2023-01-13 10:56:58
LastEditors: lin
LastEditTime: 2023-01-13 11:17:40
'''
import queue
import cnblogs_spider
from loguru import logger
import threading


# 生产者生产任务
def do_craw(url_queue:queue.Queue,html_queue:queue.Queue):

    while True:
        url = url_queue.get()
        html = cnblogs_spider.craw(url)
        logger.info("生产者：{},爬取的连接是:{},url_queue.size={}".format(
            threading.current_thread().name,
            url,
            url_queue.qsize()
            ))
        html_queue.put(html)

# 消费者消费任务
def do_parse(html_queue:queue.Queue,fout):
    while True:
        html = html_queue.get()
        results = cnblogs_spider.parse(html)
        for item in results:
            logger.info("item是:{}".format(item))
            fout.write(str(item) + '\n')
        logger.info("消费者:{},results.size:{},html_queue.size={}".format(
            threading.current_thread().name,
            len(results),
            html_queue.qsize()
        ))


if __name__ == "__main__":
    url_queue = queue.Queue()
    html_queue = queue.Queue()

    for url in cnblogs_spider.urls:
        url_queue.put(url)
    
    # 生产者开启3个线程
    for id in range(3):
        task = threading.Thread(target=do_craw,args=(url_queue,html_queue),name="craw{}".format(id))
        task.start()
    
    # 消费者开启2个线程
    # 把消费的任务写到文件中
    fout = open("./02.data.txt","w")
    for id in range(2):
        task = threading.Thread(target=do_parse,args=(html_queue,fout),name="parse{}".format(id))
        task.start()
    